from fbprophet import Prophet, forecaster
import matplotlib as plt
from matplotlib.pyplot import savefig
from numpy import add
from numpy.lib.type_check import real
import pandas as pd
from fbprophet.plot import add_changepoints_to_plot
import datetime
import time
from fbprophet.plot import plot_plotly
import plotly.offline as py
import xlrd,xlwt,xlsxwriter
from multiprocessing import Process
from multiprocessing import JoinableQueue
import csv
'''
train_raw=pd.read_csv(path,usecols=List,encoding="utf-8")#原始的训练集文件
test_raw=pd.read_csv(path,usecols=List,encoding="utf-8")#原始的测试集文件

train_data=pd.dataframe()#输入给模型的某个商品的训练集部分（占该商品数据的前85%）
test_data=pd.dataframe()#输入给模型的某个商品的测试集部分（占该商品数据的前15%）
WMAPE#输出的训练得分
'''
def get_holidays():
    Christmas = pd.DataFrame({#圣诞节
    'holiday': 'Christmas',
    'ds': pd.to_datetime(['2019-12-25', '2020-12-25','2021-12-25']),
    'lower_window': 0,'upper_window': 1,
    })
    Valentines = pd.DataFrame({
    'holiday': 'Valentine\'s Day',
    'ds': pd.to_datetime(['2019-02-14', '2020-02-14', '2021-02-14']),
    'lower_window': 0,'upper_window': 1,
    })
    Childrens = pd.DataFrame({
    'holiday': 'Children\'s Day',
    'ds': pd.to_datetime(['2019-06-01', '2020-06-01', '2021-06-01']),
    'lower_window': 0,'upper_window': 1,
    })
    Teachers = pd.DataFrame({
    'holiday': 'Teacher\'s Day',
    'ds': pd.to_datetime(['2019-09-10', '2020-09-10', '2021-09-10']),
    'lower_window': 0,'upper_window': 1,
    })
    TreePlanting = pd.DataFrame({
    'holiday': 'TreePlanting Day',
    'ds': pd.to_datetime(['2019-03-12', '2020-03-12', '2021-03-12']),
    'lower_window': 0,'upper_window': 1,
    })
    Qixi = pd.DataFrame({
    'holiday': 'Qixi Day',
    'ds': pd.to_datetime(['2019-08-07', '2020-08-25', '2021-08-14']),
    'lower_window': 0,'upper_window': 1,
    })
    holidays = pd.concat((Christmas,Valentines,Childrens,Teachers,TreePlanting,Qixi))
    return holidays

def score(data):
    up=0;down=0
    for i in range(len(data)):
        y=data.iloc[i,1];yhat=data.iloc[i,-1]
        up+=abs(y-yhat)#分子
        down+=y#分母
    return up,down

def get_div(train):#获取商品之间的分界
    div=[]
    div.append(0)
    res=train.iloc[0,1]
    for i in range(len(train)):
        if train.iloc[i,1]!=res:
            res=train.iloc[i,1]
            div.append(i)
    return div

def data_clean(data):
    data_zscore=data.copy()
    col='y'
    data_col=data[col]
    z_score=(data_col-data_col.mean())/data_col.std()
    data_zscore[col]=z_score.abs()>2.2
    data_drop=data[data_zscore['y']==False]
    data_drop=data_drop.reset_index(drop=True)
    return data_drop

def get_diff_day(train_data,test_data,begin,end):
    begin_y=pd.to_datetime(str(train_data['ds'][begin])).year
    begin_m=pd.to_datetime(str(train_data['ds'][begin])).month
    begin_d=pd.to_datetime(str(train_data['ds'][begin])).day
    end_y=pd.to_datetime(str(test_data['sale_date'][end])).year
    end_m=pd.to_datetime(str(test_data['sale_date'][end])).month
    end_d=pd.to_datetime(str(test_data['sale_date'][end])).day#计算测试集相差的天数,因为中间可能有缺失
    #print('begin:{} end:{}'.format(begin,end))
    #print(begin_y,begin_m,begin_d)
    #print(end_y,end_m,end_d)
    t=(datetime.date(end_y,end_m,end_d)-datetime.date(begin_y,begin_m,begin_d)).days
    #print('相隔: {} '.format(t+1))
    return t+1

def get_train_test(q1,q2,train,test):
    while True:
        id=q1.get()
        print('正在生成 {} 的数据集\n'.format(id))
        if id!=None:
            train_data=pd.DataFrame()
            test_data=pd.DataFrame()
            index_list=list(train[train['goodsid']==id].index)

            test_index_list=list(test[test['goodsid']==id].index)#获取测试集中对应商品ID的索引
            test_withid=test.iloc[test_index_list]#获取训练集中对应id的数据
            res_test=test_withid.sort_values(by="sale_date")
            res_test=res_test.reset_index(drop=True)#把索引按从0递增重排方便索引
            len_test=len(res_test)
            
            #print('相差天数: {}'.format(diff_test))

            '''
            因为要严格按照test文件输出，因此不需要对日期进行重排，但是由于我们要计算test中起止时间之间的天数来生成预测，所以先排个序
            '''
            
            data_raw=train.iloc[index_list]
            data_raw=data_raw.sort_values(by="sale_date")#因为训练集日期可能没按顺序，所以需要重新排一下

            data_raw=data_raw.reset_index(drop=True)#重排下标的单个商品总的数据
            del data_raw['goodsid']#把goodsid列去掉方便训练
            data_raw.columns=['ds','y']#给数据重命名为ds和y
            '''
            #这部分用于在数据集上进行数据的划分，生成result时不需要，但是数据清洗是必要的
            n_train_raw=int(len(data_raw)*0.85)
            n_test_raw=len(data_raw)-n_train_raw#数据清洗前划分训练、测试集数据个数
            begin_raw=0
            mid_raw=n_train_raw
            end_raw=n_train_raw+n_test_raw#数据集分段点

            train_data=data_raw.iloc[begin_raw:mid_raw]
            test_data=data_raw.iloc[mid_raw:end_raw]#获取清洗前的测试集和训练集
            train_data=data_clean(train_data)#对训练集进行数据清理
            train_data=train_data.reset_index(drop=True)#重排清洗后训练集商品索引
            test_data=test_data.reset_index(drop=True)#重排清洗后测试集商品索引
            '''

            train_data=data_clean(data_raw)#生成result时数据清洗是必要的

            diff_test=get_diff_day(train_data,res_test,len(train_data)-1,len_test-1)#test里商品最早最晚之间差的天数，用来预测
            n_train=len(train_data)
            #n_test=n_test_raw#测试集数量没有变化
            '''计算测试集需要的实际天数'''
            #t=get_diff_day(test_data,0,n_test-1,'ds')#自己训练时计算自己划分的测试集上的日期之差

            train_data['cap']=train_data['y'].max()+5#设置logistic预测的上下限
            train_data['floor']=0

            holidays=get_holidays()#获取专有节假日
            #prophet=Prophet(growth='linear',holidays=holidays,changepoint_prior_scale=0.008,interval_width=0.95,n_changepoints=30)#得分0.783572
            #prophet=Prophet(growth='linear',holidays=holidays,interval_width=0.95,n_changepoints=30)#得分0.803037
            prophet= Prophet(interval_width=0.95,growth='linear',changepoint_range=0.9,changepoint_prior_scale=0.008,holidays=holidays,holidays_prior_scale=0.05,n_changepoints=30)#得分0.775529
            #prophet= Prophet(interval_width=0.9,growth='logistic',changepoint_range=0.9,changepoint_prior_scale=0.008,holidays=holidays,holidays_prior_scale=0.05,n_changepoints=30,seasonality_mode='multiplicative')#得分0.821031
            #prophet= Prophet(interval_width=0.95,growth='linear',changepoint_range=0.9,changepoint_prior_scale=0.008,holidays=holidays,holidays_prior_scale=0.05,n_changepoints=40)#得分0.776811
            prophet.add_country_holidays(country_name='CN')#添加国家CH统一节假日

            prophet.add_seasonality(name='quarterly', period=91.25, fourier_order=8, mode='multiplicative',prior_scale=0.1)#每季度
            prophet.add_seasonality(name='monthly', period=30.5, fourier_order=20,mode='multiplicative',prior_scale=0.1) # 每⽉
            prophet.add_seasonality(name='weekly', period=7, fourier_order=3,mode='multiplicative',prior_scale=0.1) # 每周
                    
            prophet.fit(train_data)
            #future=prophet.make_future_dataframe(periods=t)
            future=prophet.make_future_dataframe(periods=diff_test)
            future['cap']=train_data['y'].max()+5
            future['floor']=0

            forecast=prophet.predict(future)#预测
            
            #real_and_pre=pd.merge(test_data,forecast,how='inner',on='ds')
            #----------------------------
            '''
            #这部分用于自己在训练集上的操作
            up,down=score(real_and_pre)
            fig=prophet.plot(forecast)
            fig.savefig('G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/fig3/fig'+str(id)+'.png')
            dic={'up':up,'down':down}
            #print('ID为 {} 的商品训练得分为: {} '.format(id,up/down))
            with open('G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/train_score3.txt','a') as f:
                f.write('ID为 {} 的商品训练得分为: {} \nup= {} down= {}\n'.format(id,up/down,up,down))
            '''
            #--------------------------
            dic={'forecast':forecast,'id':id}
            q2.put(dic)
        else:
            break

def output(q2,result,ans):
    while True:
        dic=q2.get()
        if dic!=None:
            '''
            #自己训练的时候生成分数用的，我们没测试集，所以没用
            ans_up=ans_up+dic['up']
            ans_down=ans_down+dic['down']
            '''
            forecast=dic['forecast']
            id=dic['id']
            forecast=forecast.iloc[0:,[0,-1]]
            forecast['goodsid']=list(id for i in range(len(forecast)))
            forecast.columns=['sale_date','sales_qty','goodsid']
            ans=pd.concat([ans,forecast])
            #print(len(ans))
        else:
            print('我已经break了')
            break
    result=pd.merge(result,ans,how='left',on=['sale_date','goodsid'])
    result.to_csv('G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/predict_code/prophet_result.csv',header=True)
    

def main():
    n=4#进程数
    begin_time = time.time()
    q1=JoinableQueue();q2=JoinableQueue()
    ans=pd.DataFrame()
    Mappers=[]
    Reduce=[]
    k=0;WMAPE=0
    up=0;down=0

    path='G:/作业/大三作业/大三下/人工智能和深度学习/预测赛/预测组赛题/预测组赛题/'

    goodsid=pd.read_csv(path+'goods_ch.csv',usecols=[0],encoding="utf-8")
    train_raw=pd.read_csv(path+'train.csv',usecols=[0,1,2],encoding="utf-8")
    test_raw=pd.read_csv(path+'test.csv',usecols=[0,1,2],encoding="utf-8")
    result=test_raw.iloc[0:,[0,1]]
    #d=pd.read_excel

    train=train_raw.copy();test=test_raw.copy()
    train['sale_date']=train['sale_date'].apply(pd.to_datetime)
    train=train.iloc[0:,[0,1,2]]
    test['sale_date']=test['sale_date'].apply(pd.to_datetime)#test文件的saledate也需要转成datetime格式方便后续处理
    result['sale_date']=result['sale_date'].apply(pd.to_datetime)#方便output函数里面的merge

    for i in goodsid['goodsid']:
        #k+=1
        q1.put(i)
        #if(k==2):break

    for i in range(n):q1.put(None)

    for i in range(n):#产生每个商品训练集和测试集加进q2的进程4个
        Map=Process(target=get_train_test,args=(q1,q2,train,test))
        Mappers.append(Map)
    Reduce=Process(target=output,args=(q2,result,ans))#从q2取训练测试集计算结果,传给output进行result输出
    Reduce.daemon=True

    for c in Mappers:
        c.start()
    Reduce.start()

    for c in Mappers:
        c.join()
    q2.put(None)
    Reduce.join()

    #print(ans.tail())
    finish_time = time.time()

    print('用时: {} seconds'.format(finish_time-begin_time))

if __name__=='__main__':main()